#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import json
import time
import random
import pymysql
import logging
from bs4 import BeautifulSoup
from datetime import datetime
from urllib.parse import urljoin

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# 数据库配置信息
DB_CONFIG = {
    'host': 'localhost',
    'port': 3306,
    'user': 'root',
    'password': '13177936413qq',
    'db': 'keep_data',
    'charset': 'utf8mb4'
}

# 伪装请求头
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 KEEP/2.51.0',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Connection': 'keep-alive',
    'Referer': 'https://www.gotokeep.com/',
    'X-App-Version': '2.51.0',
    'X-Device-Id': ''.join(random.choice('0123456789abcdef') for _ in range(40)),
    'X-Platform': 'ios',
    'X-Device-Model': 'iPhone12,1'
}

class KeepCrawler:
    def __init__(self):
        """初始化爬虫和数据库连接"""
        self.conn = None
        self.cursor = None
        self.session = requests.Session()
        self.session.headers.update(HEADERS)
        self.base_url = "https://www.gotokeep.com"
        self.api_base_url = "https://api.gotokeep.com"
        
        # 增加重试和超时配置
        self.max_retries = 3
        self.timeout = 10
        
    def init_database(self):
        """初始化数据库：连接数据库并创建表结构"""
        try:
            # 连接数据库
            self.conn = pymysql.connect(**DB_CONFIG)
            self.cursor = self.conn.cursor()
            
            # 创建表结构
            self._create_tables()
            
            logger.info("数据库初始化成功!")
            return True
        except Exception as e:
            logger.error(f"数据库初始化失败: {e}")
            if self.conn:
                self.conn.rollback()
            return False
            
    def _create_tables(self):
        """创建数据表"""
        # 创建用户表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `users` (
            `user_id` VARCHAR(50) PRIMARY KEY,
            `username` VARCHAR(100),
            `nickname` VARCHAR(100),
            `avatar_url` VARCHAR(255),
            `gender` VARCHAR(10),
            `bio` TEXT,
            `follower_count` INT,
            `following_count` INT,
            `workout_count` INT,
            `created_at` DATETIME
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 创建运动记录表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `workouts` (
            `workout_id` VARCHAR(50) PRIMARY KEY,
            `user_id` VARCHAR(50),
            `workout_type` VARCHAR(50),
            `title` VARCHAR(255),
            `duration` INT,
            `distance` FLOAT,
            `calorie` FLOAT,
            `start_time` DATETIME,
            `end_time` DATETIME,
            `average_pace` VARCHAR(20),
            `average_heart_rate` FLOAT,
            `likes_count` INT,
            `comments_count` INT,
            `route_data` TEXT,
            `created_at` DATETIME,
            FOREIGN KEY (`user_id`) REFERENCES `users`(`user_id`) ON DELETE CASCADE
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 创建训练计划表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `training_plans` (
            `plan_id` VARCHAR(50) PRIMARY KEY,
            `title` VARCHAR(255),
            `description` TEXT,
            `difficulty` VARCHAR(20),
            `duration` VARCHAR(50),
            `category` VARCHAR(50),
            `trainer` VARCHAR(100),
            `enroll_count` INT,
            `rate_score` FLOAT,
            `cover_url` VARCHAR(255),
            `created_at` DATETIME
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        self.conn.commit()
    
    def random_sleep(self, min_sleep=1, max_sleep=3):
        """随机休眠，避免请求过快"""
        sleep_time = random.uniform(min_sleep, max_sleep)
        time.sleep(sleep_time)
    
    def _make_request(self, method, url, params=None, data=None, json_data=None):
        """封装请求方法，增加重试机制"""
        retries = 0
        while retries < self.max_retries:
            try:
                # 随机更新设备ID，避免被封
                self.session.headers.update({
                    'X-Device-Id': ''.join(random.choice('0123456789abcdef') for _ in range(40))
                })
                
                response = self.session.request(
                    method=method,
                    url=url,
                    params=params,
                    data=data,
                    json=json_data,
                    timeout=self.timeout
                )
                
                # 检查响应状态
                response.raise_for_status()
                
                # 尝试解析JSON
                result = response.json()
                
                # 检查API返回的状态
                if result.get('ok') == False:
                    logger.warning(f"API返回错误: {result.get('errorMsg', '未知错误')}")
                
                return result
            except requests.exceptions.RequestException as e:
                logger.warning(f"请求失败 ({retries+1}/{self.max_retries}): {e}")
                retries += 1
                self.random_sleep(3, 5)  # 失败后休眠时间更长
            except Exception as e:
                logger.error(f"其他错误: {e}")
                return None
                
        logger.error(f"请求失败，已达到最大重试次数")
        return None
            
    def fetch_popular_feeds(self, page=1, limit=20):
        """获取热门动态(改用动态接口)"""
        try:
            url = f"{self.api_base_url}/social/v5/feed/hot"
            params = {
                'page': page,
                'limit': limit,
                'requestId': ''.join(random.choice('0123456789abcdef') for _ in range(32))
            }
            
            data = self._make_request('GET', url, params=params)
            
            if data and data.get('ok') and 'data' in data:
                feeds = data['data'].get('entries', [])
                return feeds
            return []
        except Exception as e:
            logger.error(f"获取热门动态失败: {e}")
            return []
    
    def fetch_training_plans(self, page=1, limit=20):
        """获取热门训练计划"""
        try:
            # 使用新的课程接口
            url = f"{self.api_base_url}/coursev2/v1/homeList"
            params = {
                'page': page,
                'limit': limit
            }
            
            data = self._make_request('GET', url, params=params)
            
            if data and data.get('ok') and 'data' in data:
                plans = []
                for section in data['data'].get('sections', []):
                    if section.get('courses'):
                        plans.extend(section['courses'])
                return plans
            return []
        except Exception as e:
            logger.error(f"获取训练计划失败: {e}")
            return []
    
    def fetch_user_info(self, user_id):
        """获取用户信息"""
        try:
            url = f"{self.api_base_url}/user/v1/user/{user_id}"
            
            data = self._make_request('GET', url)
            
            if data and data.get('ok') and 'data' in data:
                return data['data']
            return None
        except Exception as e:
            logger.error(f"获取用户信息失败: {e}")
            return None
    
    def extract_workout_from_feed(self, feed):
        """从动态中提取运动数据"""
        try:
            if not feed or feed.get('type') != 'workout':
                return None
                
            workout = feed.get('workout', {})
            if not workout:
                return None
                
            # 添加作者信息
            workout['author'] = feed.get('author')
            # 添加统计信息
            workout['statistics'] = {
                'likeCount': feed.get('likeCount', 0),
                'commentCount': feed.get('commentCount', 0)
            }
            
            return workout
        except Exception as e:
            logger.error(f"提取运动数据失败: {e}")
            return None
    
    def save_user(self, user_data):
        """保存用户数据到数据库"""
        try:
            if not user_data or not user_data.get('id'):
                return False
                
            sql = """
            INSERT INTO `users` 
            (`user_id`, `username`, `nickname`, `avatar_url`, `gender`, `bio`, 
             `follower_count`, `following_count`, `workout_count`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `nickname`=%s, `avatar_url`=%s, `bio`=%s, 
            `follower_count`=%s, `following_count`=%s, `workout_count`=%s
            """
            
            now = datetime.now()
            values = (
                user_data.get('id', ''),
                user_data.get('username', ''),
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('gender', ''),
                user_data.get('bio', ''),
                user_data.get('stats', {}).get('followersCount', 0),
                user_data.get('stats', {}).get('followingCount', 0),
                user_data.get('stats', {}).get('workoutCount', 0),
                now,
                # 更新值
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('bio', ''),
                user_data.get('stats', {}).get('followersCount', 0),
                user_data.get('stats', {}).get('followingCount', 0),
                user_data.get('stats', {}).get('workoutCount', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存用户数据成功: {user_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存用户数据失败: {e}")
            self.conn.rollback()
            return False
    
    def save_workout(self, workout_data):
        """保存运动记录到数据库"""
        try:
            if not workout_data or not workout_data.get('id'):
                return False
                
            # 先检查用户是否存在，如果不存在则创建一个最小记录
            user_id = workout_data.get('author', {}).get('id')
            if user_id:
                self.cursor.execute("SELECT 1 FROM `users` WHERE `user_id` = %s", (user_id,))
                if not self.cursor.fetchone():
                    self.cursor.execute("""
                    INSERT INTO `users` (`user_id`, `nickname`, `created_at`)
                    VALUES (%s, %s, %s)
                    """, (user_id, workout_data.get('author', {}).get('nickname', ''), datetime.now()))
                    self.conn.commit()
                
            sql = """
            INSERT INTO `workouts` 
            (`workout_id`, `user_id`, `workout_type`, `title`, `duration`, `distance`, 
             `calorie`, `start_time`, `end_time`, `average_pace`, `average_heart_rate`, 
             `likes_count`, `comments_count`, `route_data`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `likes_count`=%s, `comments_count`=%s
            """
            
            now = datetime.now()
            # 处理时间戳转换，增加错误处理
            try:
                start_time = datetime.fromtimestamp(workout_data.get('startTime', 0) / 1000) if workout_data.get('startTime') else None
            except:
                start_time = None
                
            try:
                end_time = datetime.fromtimestamp(workout_data.get('endTime', 0) / 1000) if workout_data.get('endTime') else None
            except:
                end_time = None
            
            # 处理JSON序列化，增加错误处理
            try:
                route_data = json.dumps(workout_data.get('routeData', {}))
            except:
                route_data = '{}'
            
            values = (
                workout_data.get('id', ''),
                user_id,
                workout_data.get('workoutType', ''),
                workout_data.get('title', ''),
                workout_data.get('duration', 0),
                workout_data.get('distance', 0),
                workout_data.get('calorie', 0),
                start_time,
                end_time,
                workout_data.get('pace', ''),
                workout_data.get('averageHeartRate', 0),
                workout_data.get('statistics', {}).get('likeCount', 0),
                workout_data.get('statistics', {}).get('commentCount', 0),
                route_data,
                now,
                # 更新值
                workout_data.get('statistics', {}).get('likeCount', 0),
                workout_data.get('statistics', {}).get('commentCount', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存运动记录成功: {workout_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存运动记录失败: {e}")
            self.conn.rollback()
            return False
    
    def save_training_plan(self, plan_data):
        """保存训练计划到数据库"""
        try:
            if not plan_data or not plan_data.get('id'):
                return False
                
            sql = """
            INSERT INTO `training_plans` 
            (`plan_id`, `title`, `description`, `difficulty`, `duration`, `category`, 
             `trainer`, `enroll_count`, `rate_score`, `cover_url`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `title`=%s, `description`=%s, `enroll_count`=%s, `rate_score`=%s
            """
            
            now = datetime.now()
            values = (
                plan_data.get('id', ''),
                plan_data.get('name', ''),
                plan_data.get('description', ''),
                plan_data.get('difficulty', ''),
                plan_data.get('duration', ''),
                plan_data.get('category', ''),
                plan_data.get('author', {}).get('name', ''),
                plan_data.get('enrollments', 0),
                plan_data.get('rating', {}).get('score', 0),
                plan_data.get('coverUrl', ''),
                now,
                # 更新值
                plan_data.get('name', ''),
                plan_data.get('description', ''),
                plan_data.get('enrollments', 0),
                plan_data.get('rating', {}).get('score', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存训练计划成功: {plan_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存训练计划失败: {e}")
            self.conn.rollback()
            return False
    
    def run(self, max_pages=5):
        """运行爬虫程序"""
        if not self.init_database():
            logger.error("数据库初始化失败，爬虫程序终止")
            return
        
        try:
            # 爬取热门动态，从中提取运动记录
            logger.info("开始爬取热门动态...")
            for page in range(1, max_pages + 1):
                feeds = self.fetch_popular_feeds(page=page)
                logger.info(f"第{page}页，获取到{len(feeds)}条动态")
                
                for feed in feeds:
                    # 从动态中提取运动记录
                    workout = self.extract_workout_from_feed(feed)
                    
                    if workout:
                        # 先保存用户信息
                        user_data = workout.get('author')
                        if user_data and user_data.get('id'):
                            user_detail = self.fetch_user_info(user_data.get('id'))
                            if user_detail:
                                self.save_user(user_detail)
                        
                        # 保存运动记录
                        self.save_workout(workout)
                    
                    self.random_sleep()
                
                self.random_sleep(2, 5)
            
            # 爬取训练计划
            logger.info("开始爬取训练计划...")
            for page in range(1, max_pages + 1):
                plans = self.fetch_training_plans(page=page)
                logger.info(f"第{page}页，获取到{len(plans)}条训练计划")
                
                for plan in plans:
                    self.save_training_plan(plan)
                    self.random_sleep()
                
                self.random_sleep(2, 5)
            
            logger.info("爬虫程序执行完成!")
        
        except Exception as e:
            logger.error(f"爬虫程序执行过程中出错: {e}")
        
        finally:
            # 关闭数据库连接
            if self.cursor:
                self.cursor.close()
            if self.conn:
                self.conn.close()

if __name__ == "__main__":
    crawler = KeepCrawler()
    crawler.run(max_pages=3)  # 爬取3页数据 